{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "通过填充Nan值为-1，就已经能很好地区分了，不需要再额外的添加指示变量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "import pandas as pd\n",
    "import numpy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## train null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_raw = joblib.load('./scrum_data/train_data/train_raw.lz4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1 = train_raw.isnull()\n",
    "train_raw[f1] = 1\n",
    "train_raw[f1 == False] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1</th>\n",
       "      <th>f2</th>\n",
       "      <th>f3</th>\n",
       "      <th>f4</th>\n",
       "      <th>f5</th>\n",
       "      <th>f6</th>\n",
       "      <th>f7</th>\n",
       "      <th>f8</th>\n",
       "      <th>f9</th>\n",
       "      <th>f10</th>\n",
       "      <th>...</th>\n",
       "      <th>f6736</th>\n",
       "      <th>f6737</th>\n",
       "      <th>f6738</th>\n",
       "      <th>f6739</th>\n",
       "      <th>f6740</th>\n",
       "      <th>f6741</th>\n",
       "      <th>f6742</th>\n",
       "      <th>f6743</th>\n",
       "      <th>f6744</th>\n",
       "      <th>f6745</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 6745 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    f1   f2   f3   f4   f5   f6   f7   f8  f9  f10  ...    f6736  f6737  \\\n",
       "0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0   0  0.0  ...      0.0    0.0   \n",
       "1  0.0  1.0  0.0  1.0  0.0  0.0  0.0  1.0   0  0.0  ...      0.0    0.0   \n",
       "2  0.0  1.0  0.0  0.0  0.0  0.0  0.0  1.0   0  0.0  ...      0.0    0.0   \n",
       "3  0.0  1.0  0.0  0.0  0.0  0.0  0.0  1.0   0  0.0  ...      0.0    0.0   \n",
       "4  0.0  1.0  0.0  1.0  0.0  0.0  0.0  1.0   0  0.0  ...      0.0    0.0   \n",
       "\n",
       "   f6738  f6739  f6740  f6741  f6742  f6743  f6744  f6745  \n",
       "0      0    1.0    1.0    0.0      0    0.0    0.0    0.0  \n",
       "1      0    1.0    1.0    0.0      0    0.0    0.0    0.0  \n",
       "2      0    1.0    1.0    0.0      0    0.0    0.0    0.0  \n",
       "3      0    1.0    1.0    0.0      0    0.0    0.0    0.0  \n",
       "4      0    1.0    1.0    0.0      0    0.0    0.0    0.0  \n",
       "\n",
       "[5 rows x 6745 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_raw.columns = ['null_sign_'+col for col in train_raw.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>null_sign_f1</th>\n",
       "      <th>null_sign_f2</th>\n",
       "      <th>null_sign_f3</th>\n",
       "      <th>null_sign_f4</th>\n",
       "      <th>null_sign_f5</th>\n",
       "      <th>null_sign_f6</th>\n",
       "      <th>null_sign_f7</th>\n",
       "      <th>null_sign_f8</th>\n",
       "      <th>null_sign_f9</th>\n",
       "      <th>null_sign_f10</th>\n",
       "      <th>...</th>\n",
       "      <th>null_sign_f6736</th>\n",
       "      <th>null_sign_f6737</th>\n",
       "      <th>null_sign_f6738</th>\n",
       "      <th>null_sign_f6739</th>\n",
       "      <th>null_sign_f6740</th>\n",
       "      <th>null_sign_f6741</th>\n",
       "      <th>null_sign_f6742</th>\n",
       "      <th>null_sign_f6743</th>\n",
       "      <th>null_sign_f6744</th>\n",
       "      <th>null_sign_f6745</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 6745 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   null_sign_f1  null_sign_f2  null_sign_f3  null_sign_f4  null_sign_f5  \\\n",
       "0           0.0           1.0           0.0           0.0           0.0   \n",
       "1           0.0           1.0           0.0           1.0           0.0   \n",
       "2           0.0           1.0           0.0           0.0           0.0   \n",
       "3           0.0           1.0           0.0           0.0           0.0   \n",
       "4           0.0           1.0           0.0           1.0           0.0   \n",
       "\n",
       "   null_sign_f6  null_sign_f7  null_sign_f8  null_sign_f9  null_sign_f10  \\\n",
       "0           0.0           0.0           0.0             0            0.0   \n",
       "1           0.0           0.0           1.0             0            0.0   \n",
       "2           0.0           0.0           1.0             0            0.0   \n",
       "3           0.0           0.0           1.0             0            0.0   \n",
       "4           0.0           0.0           1.0             0            0.0   \n",
       "\n",
       "        ...         null_sign_f6736  null_sign_f6737  null_sign_f6738  \\\n",
       "0       ...                     0.0              0.0                0   \n",
       "1       ...                     0.0              0.0                0   \n",
       "2       ...                     0.0              0.0                0   \n",
       "3       ...                     0.0              0.0                0   \n",
       "4       ...                     0.0              0.0                0   \n",
       "\n",
       "   null_sign_f6739  null_sign_f6740  null_sign_f6741  null_sign_f6742  \\\n",
       "0              1.0              1.0              0.0                0   \n",
       "1              1.0              1.0              0.0                0   \n",
       "2              1.0              1.0              0.0                0   \n",
       "3              1.0              1.0              0.0                0   \n",
       "4              1.0              1.0              0.0                0   \n",
       "\n",
       "   null_sign_f6743  null_sign_f6744  null_sign_f6745  \n",
       "0              0.0              0.0              0.0  \n",
       "1              0.0              0.0              0.0  \n",
       "2              0.0              0.0              0.0  \n",
       "3              0.0              0.0              0.0  \n",
       "4              0.0              0.0              0.0  \n",
       "\n",
       "[5 rows x 6745 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cols = [col for col in train_raw.columns if len(train_raw[col].unique())==2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5460"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(use_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_null_sign = train_raw[use_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./scrum_data/train_data/train_null_sign.lz4']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(train_null_sign,'./scrum_data/train_data/train_null_sign.lz4',compress='lz4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_null_sign = joblib.load('./scrum_data/train_data/train_null_sign.lz4')\n",
    "null_sign_fscore = joblib.load('./scrum_phase/get_null_sign_fscore/f_score')\n",
    "null_sign_cols = list(null_sign_fscore[:90].index)\n",
    "joblib.dump(train_null_sign[null_sign_cols],'./scrum_data/train_data/train_null_sign90.lz4',compress='lz4')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## valid_null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_raw = joblib.load('./scrum_data/valid_data/valid_raw.lz4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./scrum_data/valid_data/valid_null_sign.lz4']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1 = valid_raw.isnull()\n",
    "valid_raw[f1] = 1\n",
    "valid_raw[f1 == False] = 0\n",
    "valid_raw.columns = ['null_sign_'+col for col in valid_raw.columns]\n",
    "use_cols = [col for col in valid_raw.columns if len(valid_raw[col].unique())==2]\n",
    "valid_null_sign = valid_raw[use_cols]\n",
    "joblib.dump(valid_null_sign,'./scrum_data/valid_data/valid_null_sign.lz4',compress='lz4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20000, 5460)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_null_sign.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./scrum_data/valid_data/valid_null_sign90.lz4']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_null_sign = joblib.load('./scrum_data/valid_data/valid_null_sign.lz4')\n",
    "null_sign_fscore = joblib.load('./scrum_phase/get_null_sign_fscore/f_score')\n",
    "null_sign_cols = list(null_sign_fscore[:90].index)\n",
    "joblib.dump(valid_null_sign[null_sign_cols],'./scrum_data/valid_data/valid_null_sign90.lz4',compress='lz4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
